{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Seldon and TensorFlow Serving MNIST Example\n",
    "\n",
    "This example shows how you can combine Seldon with Tensorflo Serving. We will use a Seldon Tensorflow Serving proxy model image that will forward Seldon internal microservice prediction calls out to a Tensorflow serving server.\n",
    "\n",
    "The example will use the MNIST digit classification task with the example MNIST model.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install seldon-core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import requests\n",
    "from random import randint,random\n",
    "import json\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import sys\n",
    "sys.path.append(\"../../../notebooks\")\n",
    "from visualizer import get_graph\n",
    "from seldon_core.proto import prediction_pb2\n",
    "from seldon_core.proto import prediction_pb2_grpc\n",
    "import grpc\n",
    "import tensorflow as tf\n",
    "from tensorflow.core.framework.tensor_pb2 import TensorProto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_image(arr):\n",
    "    two_d = (np.reshape(arr, (28, 28)) * 255).astype(np.uint8)\n",
    "    plt.imshow(two_d,cmap=plt.cm.gray_r, interpolation='nearest')\n",
    "    return plt\n",
    "\n",
    "def download_mnist():\n",
    "    return input_data.read_data_sets(\"MNIST_data/\", one_hot = True)\n",
    "\n",
    "def rest_predict_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/predict\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_transform_input_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/transform-input\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_transform_output_request(endpoint,data):\n",
    "    request = {\"data\":{\"ndarray\":data.tolist()}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/transform-output\",\n",
    "                data={\"json\":json.dumps(request),\"isDefault\":True})\n",
    "    return response.json()   \n",
    "\n",
    "def rest_request_ambassador(deploymentName,endpoint=\"localhost:8003\",arr=None):\n",
    "    payload = {\"data\":{\"names\":[\"a\",\"b\"],\"tensor\":{\"shape\":[1,784],\"values\":arr.tolist()}}}\n",
    "    response = requests.post(\n",
    "                \"http://\"+endpoint+\"/seldon/\"+deploymentName+\"/api/v0.1/predictions\",\n",
    "                json=payload)\n",
    "    print(response.status_code)\n",
    "    print(response.text)\n",
    "\n",
    "def grpc_request_internal(data,endpoint=\"localhost:5000\"):\n",
    "    datadef = prediction_pb2.DefaultData(\n",
    "            tftensor=tf.make_tensor_proto(data)\n",
    "        )\n",
    "\n",
    "    request = prediction_pb2.SeldonMessage(data = datadef)\n",
    "    channel = grpc.insecure_channel(endpoint)\n",
    "    stub = prediction_pb2_grpc.ModelStub(channel)\n",
    "    response = stub.Predict(request=request)\n",
    "    return response\n",
    "\n",
    "\n",
    "def gen_mnist_data(mnist):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(1)\n",
    "    chosen=0\n",
    "    gen_image(batch_xs[chosen]).show()\n",
    "    data = batch_xs[chosen].reshape((1,784))\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-d2f9601b3465>:7: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "mnist = download_mnist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create MNIST Model Repository\n",
    "You will need tensorflow installed to run these steps."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Tensorflow MNIST example model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'serving'...\n",
      "remote: Enumerating objects: 75, done.\u001b[K\n",
      "remote: Counting objects: 100% (75/75), done.\u001b[K\n",
      "remote: Compressing objects: 100% (43/43), done.\u001b[K\n",
      "remote: Total 11251 (delta 53), reused 49 (delta 32), pack-reused 11176\u001b[K\n",
      "Receiving objects: 100% (11251/11251), 3.62 MiB | 3.02 MiB/s, done.\n",
      "Resolving deltas: 100% (8109/8109), done.\n",
      "Checking connectivity... done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/tensorflow/serving.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/clive/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Training model...\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting /tmp/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting /tmp/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting /tmp/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting /tmp/t10k-labels-idx1-ubyte.gz\n",
      "2018-09-26 15:48:35.512962: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "training accuracy 0.9162\n",
      "Done training!\n",
      "Exporting trained model to b'../../../mnist-model/1'\n",
      "Done exporting!\n"
     ]
    }
   ],
   "source": [
    "!cd serving/tensorflow_serving/example && python mnist_saved_model.py --training_iteration=100000 ../../../mnist-model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Copy Model to Google Bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: MODEL_REPOSITORY_BUCKET=gs://seldon-tfserving-store\n"
     ]
    }
   ],
   "source": [
    "# CHANGE THIS TO YOUR OWN CHOSEN GOOGLE BUCKET NAME\n",
    "%env MODEL_REPOSITORY_BUCKET=gs://seldon-tfserving-store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating gs://seldon-tfserving-store/...\n",
      "ServiceException: 409 Bucket seldon-tfserving-store already exists.\n"
     ]
    }
   ],
   "source": [
    "!gsutil mb ${MODEL_REPOSITORY_BUCKET}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://mnist-model/1/saved_model.pb [Content-Type=application/octet-stream]...\n",
      "Copying file://mnist-model/1/variables/variables.data-00000-of-00001 [Content-Type=application/octet-stream]...\n",
      "Copying file://mnist-model/1/variables/variables.index [Content-Type=application/octet-stream]...\n",
      "- [3 files][ 48.6 KiB/ 48.6 KiB]                                                \n",
      "Operation completed over 3 objects/48.6 KiB.                                     \n"
     ]
    }
   ],
   "source": [
    "!gsutil cp -r mnist-model ${MODEL_REPOSITORY_BUCKET}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test From GCP Cluster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run MNIST Inference Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "serviceaccount \"tiller\" created\n",
      "clusterrolebinding \"tiller\" created\n",
      "$HELM_HOME has been configured at /home/clive/.helm.\n",
      "\n",
      "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n",
      "\n",
      "Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.\n",
      "To prevent this, run `helm init` with the --tiller-tls-verify flag.\n",
      "For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation\n",
      "Happy Helming!\n"
     ]
    }
   ],
   "source": [
    "!kubectl -n kube-system create sa tiller\n",
    "!kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller\n",
    "!helm init --service-account tiller"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deployment \"tiller-deploy\" successfully rolled out\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error: a release named seldon-core-crd already exists.\r\n",
      "Run: helm ls --all seldon-core-crd; to check the status of the release\r\n",
      "Or run: helm del --purge seldon-core-crd; to delete it\r\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-crd --name seldon-core-crd \\\n",
    "    --set usage_metrics.enabled=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "namespace \"seldon\" created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create namespace seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context \"gke_kubeflow-testing-213813_europe-west4-a_cluster-2\" modified.\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl config set-context $(kubectl config current-context) --namespace=seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core\n",
      "LAST DEPLOYED: Wed Sep 26 15:53:10 2018\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/ClusterRoleBinding\n",
      "NAME    AGE\n",
      "seldon  1s\n",
      "\n",
      "==> v1beta1/RoleBinding\n",
      "NAME        AGE\n",
      "ambassador  1s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                                DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "seldon-core-ambassador              1        1        1           0          0s\n",
      "seldon-core-seldon-apiserver        1        1        1           0          0s\n",
      "seldon-core-seldon-cluster-manager  1        1        1           0          0s\n",
      "seldon-core-redis                   1        1        1           0          0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME    SECRETS  AGE\n",
      "seldon  1        1s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "seldon-crd  1s\n",
      "\n",
      "==> v1beta1/Role\n",
      "ambassador    1s\n",
      "seldon-local  1s\n",
      "\n",
      "==> v1/RoleBinding\n",
      "NAME    AGE\n",
      "seldon  1s\n",
      "\n",
      "==> v1/Service\n",
      "NAME                          TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)                        AGE\n",
      "seldon-core-ambassador-admin  NodePort   10.19.242.55   <none>       8877:32623/TCP                 1s\n",
      "seldon-core-ambassador        NodePort   10.19.252.46   <none>       8080:32423/TCP                 1s\n",
      "seldon-core-seldon-apiserver  NodePort   10.19.240.170  <none>       8080:31470/TCP,5000:30245/TCP  1s\n",
      "seldon-core-redis             ClusterIP  10.19.248.179  <none>       6379/TCP                       1s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                                 READY  STATUS             RESTARTS  AGE\n",
      "seldon-core-ambassador-778c58bf5d-tk9hz              0/2    ContainerCreating  0         0s\n",
      "seldon-core-seldon-apiserver-6b8dbc978b-56std        0/1    ContainerCreating  0         0s\n",
      "seldon-core-seldon-cluster-manager-596d4674fd-swqs6  0/1    ContainerCreating  0         0s\n",
      "seldon-core-redis-8668565565-5g98j                   0/1    ContainerCreating  0         0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "Thank you for installing Seldon Core.\n",
      "\n",
      "Documentation can be found at https://github.com/SeldonIO/seldon-core\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core --name seldon-core \\\n",
    "        --namespace seldon \\\n",
    "        --set ambassador.enabled=true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"seldon-core-seldon-cluster-manager\" successfully rolled out\n",
      "Waiting for rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"seldon-core-seldon-apiserver\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/seldon-core-seldon-cluster-manager -n seldon\n",
    "!kubectl rollout status deploy/seldon-core-seldon-apiserver -n seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   tfserving-mnist\n",
      "LAST DEPLOYED: Wed Sep 26 15:56:41 2018\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1alpha2/SeldonDeployment\n",
      "NAME             AGE\n",
      "tfserving-mnist  0s\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install tfserving-mnist --name tfserving-mnist --namespace seldon --set tfserving.model_base_path=${MODEL_REPOSITORY_BUCKET}/mnist-model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "!helm template tfserving-mnist --name tfserving-mnist --namespace seldon --set tfserving.model_base_path=${MODEL_REPOSITORY_BUCKET}/mnist-model > mnist.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "!sed '1,2d' mnist.json > tmp.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"142pt\" height=\"171pt\"\n",
       " viewBox=\"0.00 0.00 142.00 171.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 167)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-167 138,-167 138,4 -4,4\"/>\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_0</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"8,-8 8,-155 126,-155 126,-8 8,-8\"/>\n",
       "<text text-anchor=\"middle\" x=\"67\" y=\"-139.8\" font-family=\"Times,serif\" font-size=\"14.00\">predictor&#45;0</text>\n",
       "</g>\n",
       "<!-- tfserving&#45;proxy0 -->\n",
       "<g id=\"node1\" class=\"node\"><title>tfserving&#45;proxy0</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"117.5,-124 16.5,-124 16.5,-88 117.5,-88 117.5,-124\"/>\n",
       "<text text-anchor=\"middle\" x=\"67\" y=\"-102.3\" font-family=\"Times,serif\" font-size=\"14.00\">tfserving&#45;proxy</text>\n",
       "</g>\n",
       "<!-- tfserving&#45;proxy0endpoint -->\n",
       "<g id=\"node2\" class=\"node\"><title>tfserving&#45;proxy0endpoint</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"67\" cy=\"-34\" rx=\"32.4942\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"67\" y=\"-30.3\" font-family=\"Times,serif\" font-size=\"14.00\">REST</text>\n",
       "</g>\n",
       "<!-- tfserving&#45;proxy0&#45;&gt;tfserving&#45;proxy0endpoint -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>tfserving&#45;proxy0&#45;&gt;tfserving&#45;proxy0endpoint</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M67,-87.6966C67,-79.9827 67,-70.7125 67,-62.1124\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"70.5001,-62.1043 67,-52.1043 63.5001,-62.1044 70.5001,-62.1043\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f7485a2da58>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_graph(\"tmp.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\r\n",
      "\u001b[04m\u001b[31;01m#\u001b[39;49;00m \u001b[04m\u001b[31;01mS\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mu\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mc\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01m:\u001b[39;49;00m \u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01mf\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mv\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mg\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m_\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01mf\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mv\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mg\u001b[39;49;00m\u001b[04m\u001b[31;01m_\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01my\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m.\u001b[39;49;00m\u001b[04m\u001b[31;01mj\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\r\n",
      "{\r\n",
      "    \u001b[34;01m\"apiVersion\"\u001b[39;49;00m: \u001b[33m\"machinelearning.seldon.io/v1alpha2\"\u001b[39;49;00m,\r\n",
      "    \u001b[34;01m\"kind\"\u001b[39;49;00m: \u001b[33m\"SeldonDeployment\"\u001b[39;49;00m,\r\n",
      "    \u001b[34;01m\"metadata\"\u001b[39;49;00m: {\r\n",
      "        \u001b[34;01m\"labels\"\u001b[39;49;00m: {\r\n",
      "            \u001b[34;01m\"app\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m\r\n",
      "        },\r\n",
      "        \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tfserving-mnist\"\u001b[39;49;00m,\r\n",
      "\t\u001b[34;01m\"namespace\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m\t\r\n",
      "    },\r\n",
      "    \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n",
      "        \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tf-mnist\"\u001b[39;49;00m,\r\n",
      "        \u001b[34;01m\"predictors\"\u001b[39;49;00m: [\r\n",
      "            {\r\n",
      "                \u001b[34;01m\"componentSpecs\"\u001b[39;49;00m: [{\r\n",
      "                    \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n",
      "                        \u001b[34;01m\"containers\"\u001b[39;49;00m: [\r\n",
      "                            {\r\n",
      "                                \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/tfserving-proxy:0.1\"\u001b[39;49;00m,\r\n",
      "                                \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tfserving-proxy\"\u001b[39;49;00m\r\n",
      "                            },\r\n",
      "\t\t\t    {\r\n",
      "\t\t\t\t\u001b[34;01m\"args\"\u001b[39;49;00m: [\r\n",
      "\t\t\t\t    \u001b[33m\"/usr/bin/tensorflow_model_server\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t    \u001b[33m\"--port=8000\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t    \u001b[33m\"--model_name=mnist-model\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t    \u001b[33m\"--model_base_path=gs://seldon-tfserving-store/mnist-model\"\u001b[39;49;00m\r\n",
      "\t\t\t\t],\r\n",
      "\t\t\t\t\u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"gcr.io/kubeflow-images-public/tensorflow-serving-1.7:v20180604-0da89b8a\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mnist-model\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t\u001b[34;01m\"ports\"\u001b[39;49;00m: [\r\n",
      "\t\t\t\t    {\r\n",
      "\t\t\t\t\t\u001b[34;01m\"containerPort\"\u001b[39;49;00m: \u001b[34m8000\u001b[39;49;00m,\r\n",
      "\t\t\t\t\t\u001b[34;01m\"protocol\"\u001b[39;49;00m: \u001b[33m\"TCP\"\u001b[39;49;00m\r\n",
      "\t\t\t\t    }\r\n",
      "\t\t\t\t],\r\n",
      "\t\t\t\t\u001b[34;01m\"resources\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t    \u001b[34;01m\"limits\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t\t\u001b[34;01m\"cpu\"\u001b[39;49;00m: \u001b[33m\"4\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t\t\u001b[34;01m\"memory\"\u001b[39;49;00m: \u001b[33m\"4Gi\"\u001b[39;49;00m\r\n",
      "\t\t\t\t    },\r\n",
      "\t\t\t\t    \u001b[34;01m\"requests\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t\t\u001b[34;01m\"cpu\"\u001b[39;49;00m: \u001b[33m\"1\"\u001b[39;49;00m,\r\n",
      "\t\t\t\t\t\u001b[34;01m\"memory\"\u001b[39;49;00m: \u001b[33m\"1Gi\"\u001b[39;49;00m\r\n",
      "\t\t\t\t    }\r\n",
      "\t\t\t\t},\r\n",
      "\t\t\t\t\u001b[34;01m\"securityContext\"\u001b[39;49;00m: {\r\n",
      "\t\t\t\t    \u001b[34;01m\"runAsUser\"\u001b[39;49;00m: \u001b[34m1000\u001b[39;49;00m\r\n",
      "\t\t\t\t}\r\n",
      "\t\t\t    }\r\n",
      "\t\t\t],\r\n",
      "\t\t\t\u001b[34;01m\"terminationGracePeriodSeconds\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m\r\n",
      "\t\t    }\r\n",
      "\t\t}],\r\n",
      "                \u001b[34;01m\"graph\"\u001b[39;49;00m: {\r\n",
      "\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"tfserving-proxy\"\u001b[39;49;00m,\r\n",
      "\t\t    \u001b[34;01m\"endpoint\"\u001b[39;49;00m: { \u001b[34;01m\"type\"\u001b[39;49;00m : \u001b[33m\"REST\"\u001b[39;49;00m },\r\n",
      "\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"MODEL\"\u001b[39;49;00m,\r\n",
      "\t\t    \u001b[34;01m\"children\"\u001b[39;49;00m: [],\r\n",
      "\t\t    \u001b[34;01m\"parameters\"\u001b[39;49;00m:\r\n",
      "\t\t    [\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"grpc_endpoint\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"localhost:8000\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"model_name\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"mnist-model\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"model_output\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"scores\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"model_input\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"images\"\u001b[39;49;00m\r\n",
      "\t\t\t},\r\n",
      "\t\t\t{\r\n",
      "\t\t\t    \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"signature_name\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"STRING\"\u001b[39;49;00m,\r\n",
      "\t\t\t    \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"predict_images\"\u001b[39;49;00m\r\n",
      "\t\t\t}\r\n",
      "\t\t    ]\r\n",
      "\t\t},\r\n",
      "                \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mnist-tfserving\"\u001b[39;49;00m,\r\n",
      "                \u001b[34;01m\"replicas\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m\r\n",
      "            }\r\n",
      "        ]\r\n",
      "    }\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!pygmentize mnist.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Port forward Ambassador**\n",
    "\n",
    "```\n",
    "kubectl port-forward $(kubectl get pods -n seldon -l service=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADfRJREFUeJzt3X+I1Pedx/HX+7w2MdsSNK6ppObWNNLcEji9TOTAcNGUlHgIWoxSIXUPits/DJxJkQQhmj+yEI60vfxxGLYXqSs2bdeaiwGTawgHuYZSnIRgont3BtlYT9ldSUmtEUri+/7Y75aN2fnMON/vzHfW9/MBMjPf9/fHm0le+52Zz3fmY+4uAPH8RdkNACgH4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/ENRftvNgCxYs8J6ennYeEghldHRU58+ft0bWzRV+M3tA0rOS5kj6N3d/OrV+T0+PqtVqnkMCSKhUKg2v2/TLfjObI+lfJa2R1Ctps5n1Nrs/AO2V5z3/Cknvu/spd/+TpJ9JWldMWwBaLU/4b5H0u2mPz2TLPsPM+s2sambViYmJHIcDUKQ84Z/pQ4XPfT/Y3QfdveLule7u7hyHA1CkPOE/I2nxtMdflXQ2XzsA2iVP+I9KWmpmS8zsi5K+LelwMW0BaLWmh/rc/RMze1jSf2hyqG+vux8vrDMALZVrnN/dj0g6UlAvANqIy3uBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IKtcsvWY2KumCpE8lfeLulSKawmedOnUqWX/uuedq1kZHR5PbHjx4MFm/++67k/W77rorWT906FDN2ty5c5PbfvDBB8l6Pe5es7Z169bktoODg7mOPRvkCn9mtbufL2A/ANqIl/1AUHnD75J+ZWZvmVl/EQ0BaI+8L/tXuvtZM1so6TUz+293f2P6CtkfhX5JuvXWW3MeDkBRcp353f1sdjsu6UVJK2ZYZ9DdK+5e6e7uznM4AAVqOvxm1mVmX566L+mbkt4rqjEArZXnZf/Nkl40s6n9/NTdXy2kKwAt13T43f2UpL8psJewxsfHk/Xbb789Wc/+ADel3rbVajVZP3r0aK79t2rbeur1HQFDfUBQhB8IivADQRF+ICjCDwRF+IGgivhWH3J66qmnym4hnCVLlpTdQuk48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIzzd4Dh4eFc28+fP79mbdu2bcltN2zYkOvYzzzzTLI+MjJSs9bb25vr2CdOnEjWK5XavyQ/MDCQ69jXAs78QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4/zXgNSYdX9/a6dQ3LdvX7J+8eLFmrWurq5cx07tu4j9X+s48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUHXH+c1sr6S1ksbd/c5s2XxJP5fUI2lU0iZ3/33r2pzd9u/fn6yPjY0l6+6erN9www1X3VO7tHKsnXH8fBo58/9E0gNXLHtc0uvuvlTS69ljALNI3fC7+xuSPrxi8TpJU5d27ZO0vuC+ALRYs+/5b3b3c5KU3S4sriUA7dDyD/zMrN/MqmZWnZiYaPXhADSo2fCPmdkiScpux2ut6O6D7l5x90p3d3eThwNQtGbDf1hSX3a/T9JLxbQDoF3qht/MXpD0G0lfN7MzZvZdSU9Lut/MTkq6P3sMYBapO87v7ptrlL5RcC9hmVmu7fksBc3gCj8gKMIPBEX4gaAIPxAU4QeCIvxAUPx09zVgaGioZu2RRx5pYyeYTTjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPO3wdq1a5P1m266KVk/f/58sn7y5MmateXLlye3zevRRx9N1leuXFmzdttttxXdDq4CZ34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIpx/jaYN29esj5nzpxc+7906VLN2rFjx3Ltu56+vr5kfenSpTVrAwMDyW0ffPDBpnpCYzjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQdcf5zWyvpLWSxt39zmzZk5K2SpqaG3qnux9pVZOz3fDwcLJe7/v67l5kO4Wq11vqtwZ27NiR3Db1WwCStGjRomQdaY2c+X8i6YEZlv/I3Zdl/wg+MMvUDb+7vyHpwzb0AqCN8rznf9jMjpnZXjNLX78KoOM0G/49kr4maZmkc5J+UGtFM+s3s6qZVScmJmqtBqDNmgq/u4+5+6fuflnSjyWtSKw76O4Vd690d3c32yeAgjUVfjOb/jHrtyS9V0w7ANqlkaG+FyStkrTAzM5I2i1plZktk+SSRiV9r4U9AmiBuuF3980zLH6+Bb1cszZu3Jisv/LKK8n6gQMHkvXe3t6atQ0bNiS37enpSdbr2bJlS7JuZjVrp0+fTm47NDSUrD/22GPJOtK4wg8IivADQRF+ICjCDwRF+IGgCD8QFD/d3QF27dqVrNf7eex77723yHauyssvv5ysHzx4sOl9j4yMNL0t6uPMDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBMc7fAep9rTbv125b6YknnkjW84zznzhxIlm/ePFist7V1dX0sSPgzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQTHOj1zyjOPXk/pJcolx/Lw48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUHXH+c1ssaQhSV+RdFnSoLs/a2bzJf1cUo+kUUmb3P33rWt19rp06VKy/tFHH+Xa/4033lizNnfu3Fz7rufIkSPJurvXrF133XXJbe+7776mekJjGjnzfyLp++7+15L+TtI2M+uV9Lik1919qaTXs8cAZom64Xf3c+7+dnb/gqQRSbdIWidpX7baPknrW9UkgOJd1Xt+M+uRtFzSbyXd7O7npMk/EJIWFt0cgNZpOPxm9iVJv5S03d3/cBXb9ZtZ1cyqExMTzfQIoAUaCr+ZfUGTwT/g7oeyxWNmtiirL5I0PtO27j7o7hV3r3R3dxfRM4AC1A2/mZmk5yWNuPsPp5UOS5qaPrZP0kvFtwegVRr5Su9KSd+R9K6ZvZMt2ynpaUm/MLPvSjotaWNrWpz99uzZk6zv2LEjWU8Nl0nSPffcU7O2evXq5LZ33HFHsv7qq68m69VqNVmfPHfMbNWqVcltt2zZkqwjn7rhd/dfS6r1X/AbxbYDoF24wg8IivADQRF+ICjCDwRF+IGgCD8QFD/d3Qb79+9v6f7ffPPNpmpl6+SpxyPgzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQTHO3wa7d+9O1h966KFk/eOPPy6ynbYaGBioWdu2bVsbO8GVOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM87fB+vXpOUyHh4eT9U2bNiXru3btqlnr6+urWZOk48ePJ+v1fre/3hTg27dvr1m7/vrrk9uitTjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQdcf5zWyxpCFJX5F0WdKguz9rZk9K2ippIlt1p7sfaVWj17I1a9Yk6xcuXGjZsRcuXJisr169umXHRrkaucjnE0nfd/e3zezLkt4ys9ey2o/c/ZnWtQegVeqG393PSTqX3b9gZiOSbml1YwBa66re85tZj6Tlkn6bLXrYzI6Z2V4zm1djm34zq5pZdWJiYqZVAJSg4fCb2Zck/VLSdnf/g6Q9kr4maZkmXxn8YKbt3H3Q3SvuXunu7i6gZQBFaCj8ZvYFTQb/gLsfkiR3H3P3T939sqQfS1rRujYBFK1u+M3MJD0vacTdfzht+aJpq31L0nvFtwegVRr5tH+lpO9IetfM3smW7ZS02cyWSXJJo5K+15IOAbREI5/2/1qSzVBiTB+YxbjCDwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EJS5e/sOZjYh6YNpixZIOt+2Bq5Op/bWqX1J9NasInv7K3dv6Pfy2hr+zx3crOruldIaSOjU3jq1L4nemlVWb7zsB4Ii/EBQZYd/sOTjp3Rqb53al0RvzSqlt1Lf8wMoT9lnfgAlKSX8ZvaAmf2Pmb1vZo+X0UMtZjZqZu+a2TtmVi25l71mNm5m701bNt/MXjOzk9ntjNOkldTbk2b2f9lz946Z/UNJvS02s/80sxEzO25m/5QtL/W5S/RVyvPW9pf9ZjZH0v9Kul/SGUlHJW129xNtbaQGMxuVVHH30seEzezvJf1R0pC735kt+2dJH7r709kfznnu/liH9PakpD+WPXNzNqHMoukzS0taL+kfVeJzl+hrk0p43so486+Q9L67n3L3P0n6maR1JfTR8dz9DUkfXrF4naR92f19mvyfp+1q9NYR3P2cu7+d3b8gaWpm6VKfu0RfpSgj/LdI+t20x2fUWVN+u6RfmdlbZtZfdjMzuDmbNn1q+vSFJfdzpbozN7fTFTNLd8xz18yM10UrI/wzzf7TSUMOK939byWtkbQte3mLxjQ0c3O7zDCzdEdodsbropUR/jOSFk97/FVJZ0voY0bufja7HZf0ojpv9uGxqUlSs9vxkvv5s06auXmmmaXVAc9dJ814XUb4j0paamZLzOyLkr4t6XAJfXyOmXVlH8TIzLokfVOdN/vwYUl92f0+SS+V2MtndMrMzbVmllbJz12nzXhdykU+2VDGv0iaI2mvuw+0vYkZmNltmjzbS5OTmP60zN7M7AVJqzT5ra8xSbsl/bukX0i6VdJpSRvdve0fvNXobZUmX7r+eebmqffYbe7tHkn/JeldSZezxTs1+f66tOcu0ddmlfC8cYUfEBRX+AFBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCOr/AYMJ4yvRMtjWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7484af5e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "{\n",
      "  \"meta\": {\n",
      "    \"puid\": \"faun2l1ftmp3r90cm6s5vnjo8a\",\n",
      "    \"tags\": {\n",
      "    },\n",
      "    \"routing\": {\n",
      "    }\n",
      "  },\n",
      "  \"data\": {\n",
      "    \"names\": [\"t:0\", \"t:1\", \"t:2\", \"t:3\", \"t:4\", \"t:5\", \"t:6\", \"t:7\", \"t:8\", \"t:9\"],\n",
      "    \"tensor\": {\n",
      "      \"shape\": [1, 10],\n",
      "      \"values\": [1.124886839534156E-4, 9.059235132891086E-12, 0.0028196186758577824, 0.005111460108309984, 2.414313166809734E-5, 0.6415375471115112, 1.74979015810095E-6, 1.2785285052885342E-9, 0.34967318177223206, 7.198737002909184E-4]\n",
      "    }\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "data = gen_mnist_data(mnist)\n",
    "data = data.reshape((784))\n",
    "rest_request_ambassador(\"tfserving-mnist\",endpoint=\"localhost:8003\",arr=data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analytics and Load Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core-analytics\n",
      "LAST DEPLOYED: Mon Sep 24 15:52:34 2018\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Service\n",
      "NAME                      TYPE       CLUSTER-IP     EXTERNAL-IP  PORT(S)       AGE\n",
      "alertmanager              ClusterIP  10.19.253.159  <none>       80/TCP        1s\n",
      "grafana-prom              NodePort   10.19.251.82   <none>       80:30690/TCP  1s\n",
      "prometheus-node-exporter  ClusterIP  None           <none>       9100/TCP      1s\n",
      "prometheus-seldon         ClusterIP  10.19.245.167  <none>       80/TCP        1s\n",
      "\n",
      "==> v1/Secret\n",
      "NAME                 TYPE    DATA  AGE\n",
      "grafana-prom-secret  Opaque  1     2s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "prometheus  1        2s\n",
      "\n",
      "==> v1/Job\n",
      "NAME                            DESIRED  SUCCESSFUL  AGE\n",
      "grafana-prom-import-dashboards  1        0           1s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                     DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "alertmanager-deployment  1        1        1           0          1s\n",
      "grafana-prom-deployment  1        1        1           0          1s\n",
      "prometheus-deployment    1        1        1           0          1s\n",
      "\n",
      "==> v1beta1/DaemonSet\n",
      "NAME                      DESIRED  CURRENT  READY  UP-TO-DATE  AVAILABLE  NODE SELECTOR  AGE\n",
      "prometheus-node-exporter  1        1        0      1           0          <none>         1s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                      READY  STATUS             RESTARTS  AGE\n",
      "grafana-prom-import-dashboards-wzl55      0/1    ContainerCreating  0         1s\n",
      "alertmanager-deployment-557b99ccf8-m27mm  0/1    ContainerCreating  0         1s\n",
      "grafana-prom-deployment-dd84b7788-gmdkc   0/1    ContainerCreating  0         1s\n",
      "prometheus-node-exporter-pnv9k            0/1    ContainerCreating  0         1s\n",
      "prometheus-deployment-78dd89b44f-7t62q    0/1    ContainerCreating  0         1s\n",
      "\n",
      "==> v1/ConfigMap\n",
      "NAME                       DATA  AGE\n",
      "alertmanager-server-conf   1     2s\n",
      "grafana-import-dashboards  7     2s\n",
      "prometheus-rules           4     2s\n",
      "prometheus-server-conf     1     2s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "prometheus  2s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "prometheus  2s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n",
    "    --set grafana_prom_admin_password=password \\\n",
    "    --set persistence.enabled=false \\\n",
    "    --namespace seldon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "node \"gke-cluster-2-default-pool-54fbe785-wmcb\" labeled\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   loadtest\n",
      "LAST DEPLOYED: Mon Sep 24 15:52:43 2018\n",
      "NAMESPACE: seldon\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/ReplicationController\n",
      "NAME             DESIRED  CURRENT  READY  AGE\n",
      "locust-slave-1   1        1        0      0s\n",
      "locust-master-1  1        1        0      0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME             TYPE      CLUSTER-IP     EXTERNAL-IP  PORT(S)                                       AGE\n",
      "locust-master-1  NodePort  10.19.255.204  <none>       5557:32342/TCP,5558:30493/TCP,8089:32034/TCP  0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                   READY  STATUS             RESTARTS  AGE\n",
      "locust-slave-1-zncg4   0/1    ContainerCreating  0         0s\n",
      "locust-master-1-9z8gm  0/1    ContainerCreating  0         0s\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install seldon-core-loadtesting --name loadtest  \\\n",
    "    --namespace seldon \\\n",
    "    --repo https://storage.googleapis.com/seldon-charts \\\n",
    "    --set locust.script=mnist_rest_locust.py \\\n",
    "    --set locust.host=http://tf-mnist:8000 \\\n",
    "    --set oauth.enabled=false \\\n",
    "    --set locust.hatchRate=1 \\\n",
    "    --set locust.clients=1 \\\n",
    "    --set loadtest.sendFeedback=1 \\\n",
    "    --set locust.minWait=0 \\\n",
    "    --set locust.maxWait=0 \\\n",
    "    --set replicaCount=1 \\\n",
    "    --set data.size=784\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You should port-foward the grafana dashboard\n",
    "\n",
    "```\n",
    "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3000:3000\n",
    "```\n",
    "\n",
    "You can then view an analytics dashboard inside the cluster at http://localhost:3000/dashboard/db/prediction-analytics?refresh=5s&orgId=1. Your IP address may be different. get it via minikube ip. Login with:\n",
    "\n",
    "    Username : admin\n",
    "\n",
    "    password : password (as set when starting seldon-core-analytics above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
